import re
import requests


def crawl_hot_list():
    # 新浪热榜URL
    url = "https://sina.com.cn"

    headers = {
        "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36"
    }

    try:
        response = requests.get(url, headers=headers, timeout=10)
        response.encoding = 'utf-8'
        html_content = response.text

        # 方法1：使用正则表达式提取热榜数据
        hot_list = extract_hot_list_with_regex(html_content)

        # 方法2：备用正则提取方法
        if not hot_list:
            hot_list = extract_hot_list_backup(html_content)

        return hot_list

    except Exception as e:
        print(f"爬取失败: {e}")
        return []


def extract_hot_list_with_regex(html_content):
    """使用正则表达式提取热榜数据"""
    hot_list = []

    # 正则表达式模式1：匹配热榜项
    pattern1 = r'<li><a href="(https?://[^"]+)"[^>]*>([^<]+)</a></li>'
    matches1 = re.findall(pattern1, html_content)

    # 正则表达式模式2：匹配包含data属性的热榜项
    pattern2 = r'<a[^>]*href="(https?://[^"]*weibo\.com[^"]*)"[^>]*data="\d+"[^>]*>([^<]+)</a>'
    matches2 = re.findall(pattern2, html_content)

    # 正则表达式模式3：匹配包含特定class的热榜项
    pattern3 = r'<a[^>]*href="(https?://[^"]+)"[^>]*data="\d+"[^>]*target="_blank">([^<]+)</a>'
    matches3 = re.findall(pattern3, html_content)

    # 合并所有匹配结果
    all_matches = matches1 + matches2 + matches3

    # 去重并过滤
    seen_titles = set()
    for url, title in all_matches:
        # 过滤无效标题
        if (len(title.strip()) > 2 and
                not title.startswith('点击查看更多') and
                title not in seen_titles):

            # 清理标题中的多余空格
            clean_title = re.sub(r'\s+', ' ', title.strip())

            hot_list.append({
                'title': clean_title,
                'url': url,
                'rank': len(hot_list) + 1
            })
            seen_titles.add(title)

            # 只取前20个热门话题
            if len(hot_list) >= 20:
                break

    return hot_list


def extract_hot_list_backup(html_content):
    """备用正则提取方法"""
    hot_list = []

    # 更宽松的正则模式
    pattern = r'<a\s+[^>]*href="(https?://[^"]+)"[^>]*>([^<]{5,50})</a>'
    matches = re.findall(pattern, html_content)

    # 过滤和微博、热点相关的链接
    weibo_keywords = ['weibo', '热搜', '热榜', '关注', '辟谣', '声明']
    for url, title in matches:
        if any(keyword in url or keyword in title for keyword in weibo_keywords):
            clean_title = re.sub(r'\s+', ' ', title.strip())
            if len(clean_title) > 2:
                hot_list.append({
                    'title': clean_title,
                    'url': url,
                    'rank': len(hot_list) + 1
                })

    return hot_list[:20]


def extract_from_provided_html(html_content):
    """从您提供的HTML片段中提取热榜"""
    hot_list = []

    # 直接从您提供的HTML中提取
    pattern = r'<li><a href="(https?://[^"]+)"[^>]*data="\d+"[^>]*target="_blank">([^<]+)</a></li>'
    matches = re.findall(pattern, html_content)

    for i, (url, title) in enumerate(matches, 1):
        hot_list.append({
            'rank': i,
            'title': title.strip(),
            'url': url
        })

    return hot_list


# 测试函数
def test_regex():
    """测试正则表达式"""
    # 您提供的HTML片段
    test_html = """
    <div class="blk_main_card">
        <div class="blk_main_li" tab-type="tab-cont">
            <ul class="uni-blk-list02 list-a list-0427" style="padding-top: 7px;">
                <li><a href="https://s.weibo.com/weibo?q=%E5%85%B3%E6%B3%A8%E7%AC%AC%E5%85%AB%E5%B1%8A%E8%BF%9B%E5%8D%9A%E4%BC%9A" data="0" target="_blank">关注第八届进博会</a></li>
                <li><a href="https://cj.sina.cn/articles/view/1887344341/707e96d502001opac?from=finance" data="1" target="_blank">工行建行暂停提取实体金条</a></li>
                <li id="hot_list_ad">
                <a id='hotlist_index_3' href="https://s.weibo.com/weibo?q=%E8%A2%AB%E9%AA%97%E5%88%B0%E7%BC%85%E5%8C%97%E7%94%B7%E7%94%9F%E5%88%B7%E5%88%B0%E8%87%AA%E5%B7%B1%E6%96%B0%E9%97%BB" data="2" target="_blank">被骗到缅北男生刷到自己新闻</a>
                </li>
                <li><a href="https://s.weibo.com/weibo?q=26%E5%85%83%E6%B7%B1%E5%A4%9C%E4%B8%8A%E9%97%A8%E5%BC%80%E9%94%81%E7%BB%93%E6%9E%9C%E8%A2%AB%E6%94%B61300" data="3" target="_blank">26元深夜上门开锁结果被收1300</a></li>
                <li><a href="https://tech.sina.cn/2025-11-03/detail-infwcqvp0882139.d.html" data="4" target="_blank">山姆变得像盒马</a></li>
                <li><a href="https://t.cj.sina.cn/articles/view/2660940103/9e9ab94704001kn5c?from=tech" data="5" target="_blank">巨型吊牌 防退货</a></li>
                <li><a href="https://s.weibo.com/weibo?q=T1%E4%B8%A5%E6%AD%A3%E5%A3%B0%E6%98%8E" data="6" target="_blank">T1严正声明</a></li>
                <li><a href="https://finance.sina.com.cn/jjxw/2025-11-02/doc-infvyzhm6553675.shtml" data="7" target="_blank">双鸭山辟谣出现野狼</a></li>
                <li><a href="https://s.weibo.com/weibo?q=%E6%97%A5%E6%9C%AC%E9%A6%96%E7%9B%B8%E5%AE%98%E9%82%B8%E5%8F%91%E5%B8%83%E9%AB%98%E5%B8%82%E6%97%A9%E8%8B%97%E5%A4%96%E4%BA%A4%E5%90%8D%E5%9C%BA%E9%9D%A2" data="8" target="_blank">日本首相官邸发布高市早苗外交名场面</a></li>
                <li><a href="https://k.sina.com.cn/article_1664000191_632ea0bf04001e2gw.html?from=movie" data="9" target="_blank">鬼灭之刃无限城篇定档</a></li>
                <li><a class="fe661" href="https://sinanews.sina.cn/h5/top_news_list.d.html" data="10" target="_blank">点击查看更多实时热点</a></li>
            </ul>
        </div>
    </div>
    """

    # 从测试HTML中提取
    hot_list = extract_from_provided_html(test_html)

    print("热榜数据提取结果:")
    print("=" * 50)
    for item in hot_list:
        print(f"{item['rank']:2d}. {item['title']}")
        print(f"    链接: {item['url']}")
        print()


# 主程序
if __name__ == "__main__":
    # 测试从提供的HTML中提取
    test_regex()

    print("\n" + "=" * 60)
    print("开始爬取实时热榜...")
    print("=" * 60)

    # 爬取实时热榜
    real_time_hot_list = crawl_hot_list()

    if real_time_hot_list:
        print("实时热榜爬取结果:")
        print("=" * 50)
        for item in real_time_hot_list[:10]:  # 只显示前10个
            print(f"{item['rank']:2d}. {item['title']}")
            print(f"    链接: {item['url']}")
            print()
    else:
        print("未能爬取到热榜数据")